<!DOCTYPE html>
<html lang="en-us">
  <head>

    <meta http-equiv="content-type" content="text/html; charset=utf-8">
    
<meta charset="UTF-8">
<title>Limiting Memory Usage | Elasticsearch: The Definitive Guide [2.x] | Elastic</title>
<link rel="home" href="index.html" title="Elasticsearch: The Definitive Guide [2.x]">
<link rel="up" href="docvalues-and-fielddata.html" title="Doc Values and Fielddata">
<link rel="prev" href="aggregations-and-analysis.html" title="Aggregations and Analysis">
<link rel="next" href="_fielddata_filtering.html" title="Fielddata Filtering">
<meta name="DC.type" content="Learn/Docs/Legacy/Elasticsearch/Definitive Guide/2.x">
<meta name="DC.subject" content="Elasticsearch">
<meta name="DC.identifier" content="2.x">
<meta name="robots" content="noindex,nofollow">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="viewport" content="width=device-width, initial-scale=1">
    <script src="https://cdn.optimizely.com/js/18132920325.js"></script>
    <link rel="apple-touch-icon" sizes="57x57" href="/apple-icon-57x57.png">
    <link rel="apple-touch-icon" sizes="60x60" href="/apple-icon-60x60.png">
    <link rel="apple-touch-icon" sizes="72x72" href="/apple-icon-72x72.png">
    <link rel="apple-touch-icon" sizes="76x76" href="/apple-icon-76x76.png">
    <link rel="apple-touch-icon" sizes="114x114" href="/apple-icon-114x114.png">
    <link rel="apple-touch-icon" sizes="120x120" href="/apple-icon-120x120.png">
    <link rel="apple-touch-icon" sizes="144x144" href="/apple-icon-144x144.png">
    <link rel="apple-touch-icon" sizes="152x152" href="/apple-icon-152x152.png">
    <link rel="apple-touch-icon" sizes="180x180" href="/apple-icon-180x180.png">
    <link rel="icon" type="image/png" href="/favicon-32x32.png" sizes="32x32">
    <link rel="icon" type="image/png" href="/android-chrome-192x192.png" sizes="192x192">
    <link rel="icon" type="image/png" href="/favicon-96x96.png" sizes="96x96">
    <link rel="icon" type="image/png" href="/favicon-16x16.png" sizes="16x16">
    <link rel="manifest" href="/manifest.json">
    <meta name="apple-mobile-web-app-title" content="Elastic">
    <meta name="application-name" content="Elastic">
    <meta name="msapplication-TileColor" content="#ffffff">
    <meta name="msapplication-TileImage" content="/mstile-144x144.png">
    <meta name="theme-color" content="#ffffff">
    <meta name="naver-site-verification" content="936882c1853b701b3cef3721758d80535413dbfd">
    <meta name="yandex-verification" content="d8a47e95d0972434">
    <meta name="localized" content="true">
    <meta name="st:robots" content="follow,index">
    <meta property="og:image" content="https://www.elastic.co/static/images/elastic-logo-200.png">
    <link rel="shortcut icon" href="/favicon.ico" type="image/x-icon">
    <link rel="icon" href="/favicon.ico" type="image/x-icon">
    <link rel="apple-touch-icon-precomposed" sizes="64x64" href="/favicon_64x64_16bit.png">
    <link rel="apple-touch-icon-precomposed" sizes="32x32" href="/favicon_32x32.png">
    <link rel="apple-touch-icon-precomposed" sizes="16x16" href="/favicon_16x16.png">
    <!-- Give IE8 a fighting chance -->
    <!--[if lt IE 9]>
    <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script>
    <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
    <![endif]-->
    <link rel="stylesheet" type="text/css" href="/guide/static/styles.css">
  </head>

  <!--© 2015-2021 Elasticsearch B.V. Copying, publishing and/or distributing without written permission is strictly prohibited.-->

  <body>
    <!-- Google Tag Manager -->
    <script>dataLayer = [];</script><noscript><iframe src="//www.googletagmanager.com/ns.html?id=GTM-58RLH5" height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
    <script>(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start': new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0], j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src= '//www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f); })(window,document,'script','dataLayer','GTM-58RLH5');</script>
    <!-- End Google Tag Manager -->

    <!-- Global site tag (gtag.js) - Google Analytics -->
    <script async src="https://www.googletagmanager.com/gtag/js?id=UA-12395217-16"></script>
    <script>
      window.dataLayer = window.dataLayer || [];
      function gtag(){dataLayer.push(arguments);}
      gtag('js', new Date());
      gtag('config', 'UA-12395217-16');
    </script>

    <!--BEGIN QUALTRICS WEBSITE FEEDBACK SNIPPET-->
    <script type="text/javascript">
      (function(){var g=function(e,h,f,g){
      this.get=function(a){for(var a=a+"=",c=document.cookie.split(";"),b=0,e=c.length;b<e;b++){for(var d=c[b];" "==d.charAt(0);)d=d.substring(1,d.length);if(0==d.indexOf(a))return d.substring(a.length,d.length)}return null};
      this.set=function(a,c){var b="",b=new Date;b.setTime(b.getTime()+6048E5);b="; expires="+b.toGMTString();document.cookie=a+"="+c+b+"; path=/; "};
      this.check=function(){var a=this.get(f);if(a)a=a.split(":");else if(100!=e)"v"==h&&(e=Math.random()>=e/100?0:100),a=[h,e,0],this.set(f,a.join(":"));else return!0;var c=a[1];if(100==c)return!0;switch(a[0]){case "v":return!1;case "r":return c=a[2]%Math.floor(100/c),a[2]++,this.set(f,a.join(":")),!c}return!0};
      this.go=function(){if(this.check()){var a=document.createElement("script");a.type="text/javascript";a.src=g;document.body&&document.body.appendChild(a)}};
      this.start=function(){var a=this;window.addEventListener?window.addEventListener("load",function(){a.go()},!1):window.attachEvent&&window.attachEvent("onload",function(){a.go()})}};
      try{(new g(100,"r","QSI_S_ZN_emkP0oSe9Qrn7kF","https://znemkp0ose9qrn7kf-elastic.siteintercept.qualtrics.com/WRSiteInterceptEngine/?Q_ZID=ZN_emkP0oSe9Qrn7kF")).start()}catch(i){}})();
    </script><div id="ZN_emkP0oSe9Qrn7kF"><!--DO NOT REMOVE-CONTENTS PLACED HERE--></div>
    <!--END WEBSITE FEEDBACK SNIPPET-->

    <div id="elastic-nav" style="display:none;"></div>
    <script src="https://www.elastic.co/elastic-nav.js"></script>

    <!-- Subnav -->
    <div>
      <div>
        <div class="tertiary-nav d-none d-md-block">
          <div class="container">
            <div class="p-t-b-15 d-flex justify-content-between nav-container">
              <div class="breadcrum-wrapper"><span><a href="/guide/" style="font-size: 14px; font-weight: 600; color: #000;">Docs</a></span></div>
            </div>
          </div>
        </div>
      </div>
    </div>

    <div class="main-container">
      <section id="content">
        <div class="content-wrapper">

          <section id="guide" lang="en">
            <div class="container">
              <div class="row">
                <div class="col-xs-12 col-sm-8 col-md-8 guide-section">
                  <!-- start body -->
                  <div class="page_header">
<p>
  <strong>WARNING</strong>: The 2.x versions of Elasticsearch have passed their
  <a href="https://www.elastic.co/support/eol">EOL dates</a>. If you are running
  a 2.x version, we strongly advise you to upgrade.
</p>
<p>
  This documentation is no longer maintained and may be removed. For the latest
  information, see the <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/index.html">current
  Elasticsearch documentation</a>.
</p>
</div>
<div id="content">
<div class="breadcrumbs">
<span class="breadcrumb-link"><a href="index.html">Elasticsearch: The Definitive Guide [2.x]</a></span>
»
<span class="breadcrumb-link"><a href="aggregations.html">Aggregations</a></span>
»
<span class="breadcrumb-link"><a href="docvalues-and-fielddata.html">Doc Values and Fielddata</a></span>
»
<span class="breadcrumb-node">Limiting Memory Usage</span>
</div>
<div class="navheader">
<span class="prev">
<a href="aggregations-and-analysis.html">« Aggregations and Analysis</a>
</span>
<span class="next">
<a href="_fielddata_filtering.html">Fielddata Filtering »</a>
</span>
</div>
<div class="section">
<div class="titlepage"><div><div>
<h2 class="title">
<a id="_limiting_memory_usage"></a>Limiting Memory Usage<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch-definitive-guide/edit/2.x/300_Aggregations/100_circuit_breaker_fd_settings.asciidoc">edit</a>
</h2>
</div></div></div>
<p>Once analyzed strings have been loaded into fielddata, they will sit there until
evicted (or your node crashes).  For that reason it is important to keep an eye on this
memory usage, understand how and when it loads, and how you can limit the impact on your cluster.</p>
<p>Fielddata is loaded <em>lazily</em>.  If you never aggregate on an analyzed string, you’ll
never load fielddata into memory.  Furthermore, fielddata is loaded on a per-field basis,
meaning only actively used fields will incur the "fielddata tax".</p>
<p>However, there is a subtle surprise lurking here.  Suppose your query is highly selective and
only returns 100 hits.  Most people assume fielddata is only loaded for those
100 documents.</p>
<p>In reality, fielddata will be loaded for <span class="strong strong"><strong>all</strong></span> documents in that index (for that
particular field), regardless of the query’s specificity. The logic is:
if you need access to documents X, Y, and Z for this query, you
will probably need access to other documents in the next query.</p>
<p>Unlike doc values,
the fielddata structure is not created at index time.  Instead, it is populated
on-the-fly when the query is run.  This is a potentially non-trivial operation and
can take some time.  It is cheaper to load all the values once, and keep them in
memory, than load only a portion of the total fielddata repeatedly.</p>
<p>The JVM heap is a limited resource that should be used wisely. A number of
mechanisms exist to limit the impact of fielddata on heap usage. These limits
are important because abuse of the heap will cause node instability (thanks to
slow garbage collections) or even node death (with an OutOfMemory exception).</p>
<div class="sidebar">
<div class="titlepage"><div><div>
<p class="title"><strong>Choosing a Heap Size</strong></p>
</div></div></div>
<p>There are two rules to apply when setting the Elasticsearch heap size, with
the <code class="literal">$ES_HEAP_SIZE</code> environment variable:</p>
<div class="variablelist">
<dl class="variablelist">
<dt>
<span class="term">
No more than 50% of available RAM
</span>
</dt>
<dd>
Lucene makes good use of the filesystem caches, which are managed by the
kernel. Without enough filesystem cache space, performance will suffer.
Furthermore, the more memory dedicated to the heap means less available
for all your other fields using doc values.
</dd>
<dt>
<span class="term">
No more than 32 GB
</span>
</dt>
<dd>
If the heap is less than 32 GB, the JVM can use compressed pointers, which
saves a lot of memory: 4 bytes per pointer instead of 8 bytes.
</dd>
</dl>
</div>
<p>For a longer and more complete discussion of heap sizing, see <a class="xref" href="heap-sizing.html" title="Heap: Sizing and Swapping">Heap: Sizing and Swapping</a></p>
</div>
<div class="section">
<div class="titlepage"><div><div>
<h3 class="title">
<a id="fielddata-size"></a>Fielddata Size<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch-definitive-guide/edit/2.x/300_Aggregations/100_circuit_breaker_fd_settings.asciidoc">edit</a>
</h3>
</div></div></div>
<p>The <code class="literal">indices.fielddata.cache.size</code> controls how much heap space is allocated
to fielddata.
As you are issuing queries, aggregations on analyzed strings will load into fielddata
if the field wasn’t previously loaded. If the resulting fielddata size would
exceed the specified <code class="literal">size</code>, other values will be evicted in order to make space.</p>
<p>By default, this setting is <em>unbounded</em>—Elasticsearch will never evict data
from fielddata.</p>
<p>This default was chosen deliberately: fielddata is not a transient cache. It
is an in-memory data structure that must be accessible for fast execution, and
it is expensive to build. If you have to reload data for every request,
performance is going to be awful.</p>
<p>A bounded size forces the data structure to evict data.  We will look at when
to set this value, but first a warning:</p>
<div class="warning admon">
<div class="icon"></div>
<div class="admon_content">
<p>This setting is a safeguard, not a solution for insufficient memory.</p>
<p>If you don’t have enough memory to keep your fielddata resident in memory,
Elasticsearch will constantly have to reload data from disk, and evict other
data to make space. Evictions cause heavy disk I/O  and generate a large
amount of garbage in memory, which must be garbage collected later on.</p>
</div>
</div>
<p>Imagine that you are indexing logs, using a new index every day.  Normally you
are interested in data from only the last day or two.  Although you keep older
indices around, you seldom need to query them.  However, with the default
settings, the fielddata from the old indices is never evicted! fielddata
will just keep on growing until you trip the fielddata circuit breaker (see
<a class="xref" href="_limiting_memory_usage.html#circuit-breaker" title="Circuit Breaker">Circuit Breaker</a>), which will prevent you from loading any more
fielddata.</p>
<p>At that point, you’re stuck. While you can still run queries that access
fielddata from the old indices, you can’t load any new values.  Instead, we
should evict old values to make space for the new values.</p>
<p>To prevent this scenario, place an upper limit on the fielddata by adding this
setting to the <code class="literal">config/elasticsearch.yml</code> file:</p>
<div class="pre_wrapper lang-yaml">
<pre class="programlisting prettyprint lang-yaml">indices.fielddata.cache.size:  20% <a id="CO219-1"></a><i class="conum" data-value="1"></i></pre>
</div>
<div class="calloutlist">
<table border="0" summary="Callout list">
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO219-1"><i class="conum" data-value="1"></i></a></p>
</td>
<td align="left" valign="top">
<p>Can be set to a percentage of the heap size, or a concrete
value like <code class="literal">5gb</code></p>
</td>
</tr>
</table>
</div>
<p>With this setting in place, the least recently used fielddata will be evicted
to make space for newly loaded data.</p>
<div class="warning admon">
<div class="icon"></div>
<div class="admon_content">
<p>There is another setting that you may see online:  <code class="literal">indices.fielddata.cache.expire</code>.</p>
<p>We beg that you <em>never</em> use this setting!  It will likely be deprecated in the
future.</p>
<p>This setting tells Elasticsearch to evict values from fielddata if they are older
than <code class="literal">expire</code>, whether the values are being used or not.</p>
<p>This is <em>terrible</em> for performance.  Evictions are costly, and this effectively
<em>schedules</em> evictions on purpose, for no real gain.</p>
<p>There isn’t a good reason to use this setting; we literally cannot theory-craft
a hypothetically useful situation. It exists only for backward compatibility at
the moment.  We mention the setting in this book only since, sadly, it has been
recommended in various articles on the Internet as a good performance tip.</p>
<p>It is not. Never use it!</p>
</div>
</div>
</div>

<div class="section">
<div class="titlepage"><div><div>
<h3 class="title">
<a id="monitoring-fielddata"></a>Monitoring fielddata<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch-definitive-guide/edit/2.x/300_Aggregations/100_circuit_breaker_fd_settings.asciidoc">edit</a>
</h3>
</div></div></div>
<p>It is important to keep a close watch on how much memory is being used by
fielddata, and whether any data is being evicted.  High eviction counts can
indicate a serious resource issue and a reason for poor performance.</p>
<p>Fielddata usage can be monitored:</p>
<div class="ulist itemizedlist">
<ul class="itemizedlist">
<li class="listitem">
<p>per-index using the <a href="http://www.elastic.co/guide/en/elasticsearch/reference/current/indices-stats.html" class="ulink" target="_top"><code class="literal">indices-stats</code> API</a>:</p>
<div class="pre_wrapper lang-json">
<pre class="programlisting prettyprint lang-json">GET /_stats/fielddata?fields=*</pre>
</div>
</li>
<li class="listitem">
<p>per-node using the <a href="/guide/en/elasticsearch/reference/2.4/cluster-nodes-stats.html" class="ulink" target="_top"><code class="literal">nodes-stats</code> API</a>:</p>
<div class="pre_wrapper lang-json">
<pre class="programlisting prettyprint lang-json">GET /_nodes/stats/indices/fielddata?fields=*</pre>
</div>
</li>
<li class="listitem">
Or even per-index per-node:
</li>
</ul>
</div>
<div class="pre_wrapper lang-json">
<pre class="programlisting prettyprint lang-json">GET /_nodes/stats/indices/fielddata?level=indices&amp;fields=*</pre>
</div>
<p>By setting <code class="literal">?fields=*</code>, the memory usage is broken down for each field.</p>
</div>

<div class="section">
<div class="titlepage"><div><div>
<h3 class="title">
<a id="circuit-breaker"></a>Circuit Breaker<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch-definitive-guide/edit/2.x/300_Aggregations/100_circuit_breaker_fd_settings.asciidoc">edit</a>
</h3>
</div></div></div>
<p>An astute reader might have noticed a problem with the fielddata size settings.
fielddata size is checked <em>after</em> the data is loaded.  What happens if a query
arrives that tries to load more into fielddata than available memory?  The
answer is ugly: you would get an OutOfMemoryException.</p>
<p>Elasticsearch includes a <em>fielddata circuit breaker</em> that is designed to deal
with this situation.  The circuit breaker estimates the memory requirements of
a query by introspecting the fields involved (their type, cardinality, size,
and so forth). It then checks to see whether loading the required fielddata would push
the total fielddata size over the configured percentage of the heap.</p>
<p>If the estimated query size is larger than the limit, the circuit breaker is
<em>tripped</em> and the query will be aborted and return an exception.  This happens
<em>before</em> data is loaded, which means that you won’t hit an
OutOfMemoryException.</p>
<div class="sidebar">
<div class="titlepage"><div><div>
<p class="title"><strong>Available Circuit Breakers</strong></p>
</div></div></div>
<p>Elasticsearch has a family of circuit breakers, all of which work to ensure
that memory limits are not exceeded:</p>
<div class="variablelist">
<dl class="variablelist">
<dt>
<span class="term">
<code class="literal">indices.breaker.fielddata.limit</code>
</span>
</dt>
<dd>
The <code class="literal">fielddata</code> circuit breaker limits the size of fielddata to 60% of the
heap, by default.
</dd>
<dt>
<span class="term">
<code class="literal">indices.breaker.request.limit</code>
</span>
</dt>
<dd>
The <code class="literal">request</code> circuit breaker estimates the size of structures required to
complete other parts of a request, such as creating aggregation buckets,
and limits them to 40% of the heap, by default.
</dd>
<dt>
<span class="term">
<code class="literal">indices.breaker.total.limit</code>
</span>
</dt>
<dd>
The <code class="literal">total</code> circuit breaker wraps the <code class="literal">request</code> and <code class="literal">fielddata</code> circuit
breakers to ensure that the combination of the two doesn’t use more than
70% of the heap by default.
</dd>
</dl>
</div>
</div>
<p>The circuit breaker limits can be specified in the <code class="literal">config/elasticsearch.yml</code>
file, or can be updated dynamically on a live cluster:</p>
<div class="pre_wrapper lang-js">
<pre class="programlisting prettyprint lang-js">PUT /_cluster/settings
{
  "persistent" : {
    "indices.breaker.fielddata.limit" : "40%" <a id="CO220-1"></a><i class="conum" data-value="1"></i>
  }
}</pre>
</div>
<div class="calloutlist">
<table border="0" summary="Callout list">
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO220-1"><i class="conum" data-value="1"></i></a></p>
</td>
<td align="left" valign="top">
<p>The limit is a percentage of the heap.</p>
</td>
</tr>
</table>
</div>
<p>It is best to configure the circuit breaker with a relatively conservative
value. Remember that fielddata needs to share the heap with the <code class="literal">request</code>
circuit breaker, the indexing memory buffer, the filter cache, Lucene data
structures for open indices, and various other transient data structures. For
this reason, it defaults to a fairly conservative 60%.  Overly optimistic
settings can cause potential OOM exceptions, which will take down an entire
node.</p>
<p>On the other hand, an overly conservative value will simply return a query
exception that can be handled by your application.  An exception is better
than a crash. These exceptions should also encourage you to reassess your
query: why <em>does</em> a single query need more than 60% of the heap?</p>
<div class="tip admon">
<div class="icon"></div>
<div class="admon_content">
<p>In <a class="xref" href="_limiting_memory_usage.html#fielddata-size" title="Fielddata Size">Fielddata Size</a>, we spoke about adding a limit to the size of fielddata,
to ensure that old unused fielddata can be evicted.  The relationship between
<code class="literal">indices.fielddata.cache.size</code> and <code class="literal">indices.breaker.fielddata.limit</code> is an
important one.  If the circuit-breaker limit is lower than the cache size, no data will ever be evicted.  In order for it to work properly, the
circuit breaker limit <em>must</em> be higher than the cache size.</p>
</div>
</div>
<p>It is important to note that the circuit breaker compares estimated query size
against the total heap size, <em>not</em> against the actual amount of heap memory
used.  This is done for a variety of technical reasons (for example, the heap may look
full but is actually just garbage waiting to be collected, which is hard to
estimate properly). But as the end user, this means the setting needs to be
conservative, since it is comparing against total heap, not <em>free</em> heap.
</p>
</div>

</div>
<div class="navfooter">
<span class="prev">
<a href="aggregations-and-analysis.html">« Aggregations and Analysis</a>
</span>
<span class="next">
<a href="_fielddata_filtering.html">Fielddata Filtering »</a>
</span>
</div>
</div>

                  <!-- end body -->
                </div>
                <div class="col-xs-12 col-sm-4 col-md-4" id="right_col">
                  <div id="rtpcontainer" style="display: block;">
                    <div class="mktg-promo">
                      <h3>Most Popular</h3>
                      <ul class="icons">
                        <li class="icon-elasticsearch-white"><a href="https://www.elastic.co/webinars/getting-started-elasticsearch?baymax=default&amp;elektra=docs&amp;storm=top-video">Get Started with Elasticsearch: Video</a></li>
                        <li class="icon-kibana-white"><a href="https://www.elastic.co/webinars/getting-started-kibana?baymax=default&amp;elektra=docs&amp;storm=top-video">Intro to Kibana: Video</a></li>
                        <li class="icon-logstash-white"><a href="https://www.elastic.co/webinars/introduction-elk-stack?baymax=default&amp;elektra=docs&amp;storm=top-video">ELK for Logs &amp; Metrics: Video</a></li>
                      </ul>
                    </div>
                  </div>
                </div>
              </div>
            </div>
          </section>

        </div>


<div id="elastic-footer"></div>
<script src="https://www.elastic.co/elastic-footer.js"></script>
<!-- Footer Section end-->

      </section>
    </div>

<script src="/guide/static/jquery.js"></script>
<script type="text/javascript" src="/guide/static/docs.js"></script>
<script type="text/javascript">
  window.initial_state = {}</script>
  </body>
</html>
